{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "zh2ru = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix = '../data/TED2013-v1.1/en-ru/TED2013.en-ru.'\n",
    "en2ru = {}\n",
    "with open(prefix + 'en', 'r') as fen, open(prefix + 'ru', 'r') as fru:\n",
    "    for en, ru in zip(fen, fru):\n",
    "        en = en.strip()\n",
    "        ru = ru.strip()\n",
    "        en2ru[en] = ru"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix = '../data/TED2013-v1.1/en-zh/TED2013.en-zh.'\n",
    "en2zh = {}\n",
    "with open(prefix + 'en', 'r') as fen, open(prefix + 'zh', 'r') as fzh:\n",
    "    for en, zh in zip(fen, fzh):\n",
    "        en = en.strip()\n",
    "        zh = zh.strip()\n",
    "        en2zh[en] = zh\n",
    "        if en in en2ru and not en.startswith(\"http\"):\n",
    "            zh2ru[zh] = en2ru[en]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(153016, 132345, 124560)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(en2zh), len(en2ru), len(zh2ru)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p ../data/TED2013-v1.1/zh-en-ru"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix = '../data/TED2013-v1.1/zh-en-ru/TED2013.zh-en-ru.'\n",
    "with open(prefix + 'ru', 'w') as fru, open(prefix + 'zh', 'w') as fzh:\n",
    "    for k, v in zh2ru.items():\n",
    "        if k != \"\" and v != \"\" and k != v:\n",
    "            print(k, file=fzh)\n",
    "            print(v, file=fru)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "zh2ru = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix = '../data/WMT-News-v2019/en-ru/WMT-News.en-ru.'\n",
    "en2ru = {}\n",
    "with open(prefix + 'en', 'r') as fen, open(prefix + 'ru', 'r') as fru:\n",
    "    for en, ru in zip(fen, fru):\n",
    "        en = en.strip()\n",
    "        ru = ru.strip()\n",
    "        en2ru[en] = ru"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix = '../data/WMT-News-v2019/en-zh/WMT-News.en-zh.'\n",
    "en2zh = {}\n",
    "with open(prefix + 'en', 'r') as fen, open(prefix + 'zh', 'r') as fzh:\n",
    "    for en, zh in zip(fen, fzh):\n",
    "        en = en.strip()\n",
    "        zh = zh.strip()\n",
    "        en2zh[en] = zh\n",
    "        if en in en2ru:\n",
    "            zh2ru[zh] = en2ru[en]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11955, 24807, 4497)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(en2zh), len(en2ru), len(zh2ru)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p ../data/TED2013-v1.1/zh-en-ru"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix = '../data/WMT-News-v2019/zh-en-ru/WMT-News.zh-en-ru.'\n",
    "with open(prefix + 'ru', 'w') as fru, open(prefix + 'zh', 'w') as fzh:\n",
    "    for k, v in zh2ru.items():\n",
    "        if k != \"\" and v != \"\" and k != v:\n",
    "            print(k, file=fzh)\n",
    "            print(v, file=fru)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
